/*************************************************************************
    > File Name    : WebPage.cc
    > Author       : LiuLei
    > Mail         : 2261459486@qq.com 
    > Created Time : 2022年11月29日 星期二 16时43分32秒
 ************************************************************************/

#include "../../include/WebProcessor/WebPage.h"
#include <queue>
using std::priority_queue;
using std::pair;

Simhash WebPage::_sim = Simhash();

bool operator==(const WebPage &lhs, const WebPage &rhs)
{
    return WebPage::_sim.isEqual(lhs._simValue, rhs._simValue);
}

struct WebPage::comp
{
    bool operator()(const pair<string, int> &lhs, const pair<string, int> &rhs)
    {
        return lhs.second > rhs.second;
    }
};

WebPage::WebPage(const string &doc, Configuration *config, WordSegmentation &jieba)
: _config(config)
, _jieba(jieba)
, _doc(doc)
{
    processDoc();
    calcTopK();
}

WebPage::~WebPage()
{

}

int WebPage::getDocId()
{
    return _docId;
}

string WebPage::getDoc()
{
    return _doc;
}

map<string, int> &WebPage::getWordsMap()
{
    return _wordsMap;
}

uint64_t WebPage::getSimhashValue()
{
    return _simValue;
}

vector<string> WebPage::getTopWords()
{
    return _topWords;
}

void WebPage::processDoc()
{
    size_t begin = 0, end = 0;

    begin = _doc.find("<docid>");
    end = _doc.find("</docid>");
    string docId = _doc.substr(begin + 7, end - begin - 7);
    _docId = atoi(docId.c_str());

    begin = _doc.find("<title>");
    end = _doc.find("</title>");
    _docTitle = _doc.substr(begin + 7, end - begin - 7);

    begin = _doc.find("<url>");
    end = _doc.find("</url>");
    _docLink = _doc.substr(begin + 5, end - begin - 5);

    begin = _doc.find("<content>");
    if(begin != string::npos)
    {
        end = _doc.find("</content>");
        _docContent = _doc.substr(begin + 9, end - begin - 9);

        // 统计标题的词频
        this->storeWordsMap(_docTitle);
        // 统计内容的词频
        this->storeWordsMap(_docContent);

        int length = this->getSummarySize(_docContent);
        _docSummary = _docContent.substr(0, length) + "...";
    }
    else
    {
        return ;
    }
    this->makeSimValue();
}

void WebPage::calcTopK()
{
    priority_queue<pair<string, int>, vector<pair<string, int>>, WebPage::comp> top20;

    for(auto &elem : _wordsMap)
    {
        top20.push(elem);
        // 逐渐淘汰词频小的词
        if(top20.size() > 20)
        {
            top20.pop();
        }
    }
    // 最终top20.size() <= 20
    _topWords.resize(top20.size());
    // it指向vector的末尾，将优先级队列倒序存入
    auto it = _topWords.end();
    while(!top20.empty())
    {
        // vector中越靠前的词，词频越大
        *(--it) = top20.top().first;
        top20.pop();
    }
}

void WebPage::storeWordsMap(const string &s)
{
    // 使用分词工具
    vector<string> vector = _jieba.operator()(s);

    const unordered_set<string> &tempStopWordList = _config->getStopWorlList();

    // 统计文章词频
    for(auto &elem : vector)
    {
        // 丢弃停用词
        if(tempStopWordList.find(elem) != tempStopWordList.end())
        {
            continue;
        }
        _wordsMap[elem]++;
    }

    // 停词库中不包含空格字符，需要手动删除
    auto it = _wordsMap.find(" ");
    if(it != _wordsMap.end())
    {
        _wordsMap.erase(it);
    }
}

void WebPage::makeSimValue()
{
    if(_docContent.size())
    {
        _simValue = _sim.make(_docContent);
    }
    else
    {
        _simValue = _sim.make(_docTitle);
    }
}

// 获取utf-8编码下字符占据的字节数
size_t WebPage::getUtf8ByteNum(const char byte)
{
    int byteNum = 0;
    for(size_t i = 0; i < 6; ++i)
    {
        if(byte & (1 << (7 - i))){
            ++byteNum;
        }else{
            break;
        }
    }
    return byteNum == 0 ? 1 : byteNum;
}

// 获取Summary的大小
int WebPage::getSummarySize(const string &content)
{
    int index = 0;
    int cnt = 0;
    while(cnt < 30)
    {
        index += this->getUtf8ByteNum(content[index]);
        ++cnt;
    }
    return index;
}
